{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ConvNets for CIFAR10 with TensorFlow and Keras <a class=\"tocSkip\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NumPy:1.13.1\n",
      "TensorFlow:1.4.1\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(123)\n",
    "print(\"NumPy:{}\".format(np.__version__))\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.set_random_seed(123)\n",
    "print(\"TensorFlow:{}\".format(tf.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASETSLIB_HOME = '../datasetslib'\n",
    "import sys\n",
    "if not DATASETSLIB_HOME in sys.path:\n",
    "    sys.path.append(DATASETSLIB_HOME)\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "import datasetslib\n",
    "\n",
    "datasetslib.datasets_root = os.path.join(os.path.expanduser('~'),'datasets')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the CIFAR10 Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Already exists: /home/armando/datasets/cifar10/cifar-10-python.tar.gz\n",
      "Extracting  /home/armando/datasets/cifar10/cifar-10-python.tar.gz\n"
     ]
    }
   ],
   "source": [
    "from datasetslib.cifar import cifar10\n",
    "from datasetslib import imutil\n",
    "dataset = cifar10()\n",
    "dataset.x_layout=imutil.LAYOUT_NHWC\n",
    "dataset.load_data()\n",
    "dataset.scaleX()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN with TensorFlow for CIFAR10 Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the hyperparameters\n",
    "n_filters=[32,64]\n",
    "learning_rate = 0.001\n",
    "\n",
    "# input images of shape = (n_samples,n_width,n_height,n_depth)\n",
    "x = tf.placeholder(dtype=tf.float32, name=\"x\", \n",
    "                   shape=[None, dataset.width, dataset.height, dataset.depth]\n",
    "                  ) \n",
    "# target output placeholder\n",
    "y = tf.placeholder(dtype=tf.float32, name=\"y\", shape=[None,dataset.n_classes]) \n",
    "\n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "\n",
    "# create convolutional and pooling layers\n",
    "\n",
    "# convolutional layer 1\n",
    "layer1_w = tf.Variable(tf.random_normal(shape=[4,4,dataset.depth,n_filters[0]],\n",
    "                                        stddev=0.01),\n",
    "                       name='l1_w'\n",
    "                      )\n",
    "layer1_b = tf.Variable(tf.random_normal([n_filters[0]]),name='l1_b')\n",
    "layer1_conv = tf.nn.relu(tf.nn.conv2d(x,\n",
    "                                      layer1_w,\n",
    "                                      strides=[1,1,1,1],\n",
    "                                      padding='SAME'\n",
    "                                     ) + \n",
    "                         layer1_b \n",
    "                        )\n",
    "# pooling layer 1\n",
    "layer1_pool = tf.nn.max_pool(layer1_conv,\n",
    "                             ksize=[1,2,2,1],\n",
    "                             strides=[1,2,2,1],\n",
    "                             padding='SAME')\n",
    "layer1_pool = tf.nn.dropout(layer1_pool, keep_prob)\n",
    "\n",
    "# convolutional layer 2\n",
    "layer2_w = tf.Variable(tf.random_normal(shape=[4,4,32,n_filters[1]],\n",
    "                                        stddev=0.01),\n",
    "                       name='l2_w'\n",
    "                      )\n",
    "layer2_b = tf.Variable(tf.random_normal([n_filters[1]]),\n",
    "                       name='l2_b'\n",
    "                      )\n",
    "layer2_conv = tf.nn.relu(tf.nn.conv2d(layer1_pool,\n",
    "                                      layer2_w,\n",
    "                                      strides=[1,1,1,1],\n",
    "                                      padding='SAME'\n",
    "                                     ) + \n",
    "                         layer2_b \n",
    "                        )\n",
    "# pooling layer 2\n",
    "layer2_pool = tf.nn.max_pool(layer2_conv,\n",
    "                             ksize=[1,2,2,1],\n",
    "                             strides=[1,2,2,1],\n",
    "                             padding='SAME')\n",
    "layer2_pool = tf.nn.dropout(layer2_pool, keep_prob)\n",
    "\n",
    "# fully connected layer\n",
    "layer3_w = tf.Variable(tf.random_normal(shape=[8*8*64,1024],\n",
    "                                        stddev=0.01),\n",
    "                       name='l3_w')\n",
    "layer3_b = tf.Variable(tf.random_normal([1024]),name='l3_b')\n",
    "layer3_fc = tf.nn.relu(tf.matmul(tf.reshape(layer2_pool,[-1, 8*8*64]),\n",
    "                                 layer3_w) +\n",
    "                       layer3_b\n",
    "                      )\n",
    "layer3_fc = tf.nn.dropout(layer3_fc, keep_prob)\n",
    "\n",
    "# output layer\n",
    "layer4_w = tf.Variable(tf.random_normal(shape=[1024,dataset.n_classes],\n",
    "                                        stddev=0.01),\n",
    "                       name='l4_w')\n",
    "layer4_b = tf.Variable(tf.random_normal([dataset.n_classes]),name='l4_b')\n",
    "\n",
    "layer4_out = tf.matmul(layer3_fc,layer4_w)+layer4_b\n",
    "\n",
    "model = layer4_out\n",
    "\n",
    "# loss function\n",
    "entropy = tf.nn.softmax_cross_entropy_with_logits(logits=model, \n",
    "                                                  labels=y\n",
    "                                                 )\n",
    "loss = tf.reduce_mean(entropy)\n",
    "\n",
    "# optimizer function\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0000   loss = 2.187322\n",
      "Epoch: 0001   loss = 1.823086\n",
      "Epoch: 0002   loss = 1.605821\n",
      "Epoch: 0003   loss = 1.477520\n",
      "Epoch: 0004   loss = 1.360107\n",
      "Epoch: 0005   loss = 1.255211\n",
      "Epoch: 0006   loss = 1.152037\n",
      "Epoch: 0007   loss = 1.051977\n",
      "Epoch: 0008   loss = 0.951648\n",
      "Epoch: 0009   loss = 0.861205\n",
      "Epoch: 0010   loss = 0.767075\n",
      "Epoch: 0011   loss = 0.679820\n",
      "Epoch: 0012   loss = 0.601895\n",
      "Epoch: 0013   loss = 0.517342\n",
      "Epoch: 0014   loss = 0.443260\n",
      "Epoch: 0015   loss = 0.378645\n",
      "Epoch: 0016   loss = 0.319791\n",
      "Epoch: 0017   loss = 0.267103\n",
      "Epoch: 0018   loss = 0.227229\n",
      "Epoch: 0019   loss = 0.192739\n",
      "Model Trained.\n",
      "Accuracy: 0.6295\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "dataset.y_onehot = True\n",
    "dataset.batch_size = 128\n",
    "dataset.batch_shuffle = True\n",
    "n_batches = int(dataset.n_train/dataset.batch_size)\n",
    "\n",
    "with tf.Session() as tfs:\n",
    "    tf.global_variables_initializer().run()\n",
    "    for epoch in range(n_epochs):\n",
    "        total_loss = 0.0\n",
    "        for batch in range(n_batches):\n",
    "            batch_x,batch_y = dataset.next_batch()\n",
    "            feed_dict={x:batch_x, y: batch_y, keep_prob: 1.0}\n",
    "            batch_loss,_ = tfs.run([loss,optimizer],feed_dict=feed_dict)\n",
    "            total_loss += batch_loss  \n",
    "        average_loss = total_loss / n_batches\n",
    "        print(\"Epoch: {0:04d}   loss = {1:0.6f}\".format(epoch,average_loss))\n",
    "    print(\"Model Trained.\")\n",
    "\n",
    "    predictions_check = tf.equal(tf.argmax(model,1),tf.argmax(y,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(predictions_check, tf.float32))\n",
    "    feed_dict = {x:dataset.X_test, y:dataset.Y_test, keep_prob: 1.0}\n",
    "    print(\"Accuracy:\", accuracy.eval(feed_dict=feed_dict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN with Keras for CIFAR10 Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D,MaxPooling2D, Dense, Flatten, Reshape, Dropout\n",
    "from keras.optimizers import RMSprop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 32, 32, 32)        1568      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 16, 16, 64)        32832     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1024)              4195328   \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                10250     \n",
      "=================================================================\n",
      "Total params: 4,239,978\n",
      "Trainable params: 4,239,978\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 6s 121us/step - loss: 1.6297 - acc: 0.4202\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 6s 120us/step - loss: 1.1793 - acc: 0.5852\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 1.0087 - acc: 0.6475\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 6s 121us/step - loss: 0.8868 - acc: 0.6913\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 0.7828 - acc: 0.7277\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 6s 118us/step - loss: 0.6869 - acc: 0.7595\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 0.6005 - acc: 0.7922\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 0.5290 - acc: 0.8167\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 0.4559 - acc: 0.8418\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 6s 121us/step - loss: 0.4020 - acc: 0.8611\n",
      "10000/10000 [==============================] - 1s 86us/step\n",
      "\n",
      "Test loss: 0.95237200985\n",
      "Test accuracy: 0.7126\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "keras.backend.clear_session()\n",
    "\n",
    "n_filters=[32,64]\n",
    "\n",
    "learning_rate=0.001\n",
    "\n",
    "n_epochs = 10\n",
    "batch_size = 100\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(filters=n_filters[0], \n",
    "                 kernel_size=4, \n",
    "                 padding='same',\n",
    "                 activation='relu',\n",
    "                 input_shape=(dataset.height,\n",
    "                              dataset.width,dataset.depth)\n",
    "                ) \n",
    "         )\n",
    "\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Conv2D(filters=n_filters[1], \n",
    "                 kernel_size=4, \n",
    "                 padding='same', \n",
    "                 activation='relu'\n",
    "                ) \n",
    "         )\n",
    "\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Flatten())\n",
    "\n",
    "model.add(Dense(units=1024, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Dense(units=dataset.n_classes, activation='softmax'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(lr=learning_rate),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(dataset.X_train, dataset.Y_train, batch_size=dataset.batch_size,\n",
    "          epochs=n_epochs)\n",
    "\n",
    "score = model.evaluate(dataset.X_test, dataset.Y_test)\n",
    "print('\\nTest loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
